package com.estock.crawler;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.estock.crawler.utils.ESCrawlerUtil;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

public class JqkaCrawlerController {
	private static final Logger logger = LoggerFactory.getLogger(JqkaCrawlerController.class);
	
	public static void main(String[] args) throws Exception {
		if (args.length != 1) {
			logger.info("Needed parameters: ");
			logger.info("\t numberOfCralwers (number of concurrent threads)");
			return;
		}

		int numberOfCrawlers = Integer.valueOf(args[0]);
		if (numberOfCrawlers < 1) {
			numberOfCrawlers = 1;
		}
		
		CrawlConfig config = new CrawlConfig();
		config.setCrawlStorageFolder(ESCrawlerUtil.getRootDataDir());

		/*
		 * Instantiate the controller for this crawl.
		 */
		PageFetcher pageFetcher = new PageFetcher(config);
		RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
		RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig,
				pageFetcher);
		CrawlController controller = new CrawlController(config, pageFetcher,
				robotstxtServer);

		/*
		 * For each crawl, you need to add some seed urls. These are the first
		 * URLs that are fetched and then the crawler starts following links
		 * which are found in these pages
		 */
		controller.addSeed(ESCrawlerUtil.DOMIN_10jqka);

		/*
		 * Start the crawl. This is a blocking operation, meaning that your code
		 * will reach the line after this only when crawling is finished.
		 */
		controller.startNonBlocking(JqkaCrawler.class, numberOfCrawlers);
		
		controller.waitUntilFinish();
	    logger.info("JqkaCrawler is finished.");
	}

}
